# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

from scrapy import Request
from scrapy.exceptions import DropItem
from scrapy.pipelines.images import ImagesPipeline
import pymysql

#class Images360Pipeline(object):
#    def process_item(self, item, spider):
#        return item

# 将数据存入数据库(动态构造SQL语句的方法)
class MysqlPipeline():
    def __init__(self, host, database, user, password, port):
        self.host = host
        self.database = database
        self.user = user
        self.password = password
        self.port = port
        
    @classmethod
    def from_crawler(cls, crawler):
        return cls(
                host = crawler.settings.get('MYSQL_HOST'),
                database = crawler.settings.get('MYSQL_DATABASE'),
                user = crawler.settings.get('MYSQL_USER'),
                password = crawler.settings.get('MYSQL_PASSWORD'),
                port = crawler.settings.get('MYSQL_PORT'),
            )
    
    def open_spider(self, spider):
        self.db = pymysql.connect(self.host, self.user, self.database, 
                                  self.password, self.port, charset='utf8')
        self.cursor = self.db.cursor()
        
    def close_spider(self, spider):
        self.db.close()
        
    def process_item(self, item, spider):
        data = dict(item)
        keys = ', '.join(data.keys())
        values = ', '.join(['%s'] * len(data))
        sql = 'insert into %s (%s) values (%s)' % (item.table, keys, values)
        self.cursor.execute(sql, tuple(data.values()))
        self.db.commit()
        return item


# 实现 ImagePipeline,继承 Scrapy内置的 ImagesPipeline,
class ImagePipeline(ImagesPipeline):
    # request: 当前下载对应的Request对象, 此方法用来返回保存的文件名
    def file_path(self, request, response=None, info=None):
        # url为图片的链接url
        url = request.url
        # 将url用split()拆分最后一部分作为文件名
        file_name = url.split('/')[-1]
        return file_name
    
    # 单个Item下载完成时的处理方法
    def item_completed(self, results, item, info):
        # 将下载结果results遍历,如果下载成功返回 item,如果(为空)失败,则报错并忽略
        image_paths = [x['path'] for ok, x in results if ok]
        if not image_paths:
            raise DropItem('Image Downloaded Failed')
        return item
    
    # item是爬取生成的Item对象,将其url取出生成Request对象,加入下载调度队列
    def get_media_requests(self, item, info):
        yield Request(item['url'])
